{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "双端 GRU 模型，对所有的 timestep 采用attention 模型输出。\n",
    "\n",
    "把 title 和 content 两部分分别放在两个网络中，在最后的 softmax 层之前拼起来。由于两部分的序列长度差的太多，所以跑起来会比较慢\n",
    "\n",
    "参考代码： \n",
    "\n",
    "https://github.com/yongyehuang/deep-text-classifier/blob/master/HAN_model.py 其中attention最后加权的地方应该写错了\n",
    "\n",
    "https://github.com/indiejoseph/doc-han-att/blob/master/model.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 36 ms, sys: 656 ms, total: 692 ms\n",
      "Wall time: 8.25 s\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt \n",
    "from gensim.models import KeyedVectors\n",
    "import pickle\n",
    "from tqdm import tqdm\n",
    "\n",
    "# 导入预训练好的词向量\n",
    "%time W_embedding = np.load('../data/W_embedding.npy')\n",
    "\n",
    "# 模型保存位置\n",
    "import os\n",
    "model_save_path = '../ckpt/attention-bigru-title-content/'  # 模型保存位置\n",
    "if not os.path.exists(model_save_path):\n",
    "    os.makedirs(model_save_path)\n",
    "model_save_path = model_save_path + 'bi-gru.ckpt'\n",
    "\n",
    "# summary 位置\n",
    "summary_path = '../summary/attention-bigru-title-content/'\n",
    "if not os.path.exists(summary_path):\n",
    "    os.makedirs(summary_path)\n",
    "result_path = '../result/attention-gru-title-content-256.csv'\n",
    "predict_scores_path = '../scores/attention-gru-title-content-256.npy' \n",
    "\n",
    "# ##################### config ######################\n",
    "n_step1 = max_len1 = 30           # 句子长度\n",
    "n_step2 = max_len2 = 100\n",
    "input_size = embedding_size = 256       # 字向量长度\n",
    "n_class = 1999\n",
    "hidden_size = 256    # 隐含层节点数\n",
    "n_layer = 2        # bi-gru 层数\n",
    "l2_lambda = 1e-4\n",
    "max_grad_norm = 10.0  # 最大梯度（超过此值的梯度将被裁剪）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "result_path = '../result/attention-gru-title-content-256.csv'\n",
    "predict_scores_path = '../scores/attention-gru-title-content-256.npy' "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished creating the bi-gru model.\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "config = tf.ConfigProto()\n",
    "config.gpu_options.allow_growth = True\n",
    "sess = tf.Session(config=config)\n",
    "from tensorflow.contrib import rnn\n",
    "import tensorflow.contrib.layers as layers\n",
    "\n",
    "'''\n",
    "双端 GRU，知乎问题多标签分类。\n",
    "'''\n",
    "\n",
    "lr = tf.placeholder(tf.float32)\n",
    "keep_prob = tf.placeholder(tf.float32, [])\n",
    "batch_size = tf.placeholder(tf.int32, [])  # 注意类型必须为 tf.int32\n",
    "\n",
    "def weight_variable(shape, name):\n",
    "    \"\"\"Create a weight variable with appropriate initialization.\"\"\"\n",
    "    initial = tf.truncated_normal(shape, stddev=0.1)\n",
    "    return tf.Variable(initial, name=name)\n",
    "\n",
    "def bias_variable(shape, name):\n",
    "    \"\"\"Create a bias variable with appropriate initialization.\"\"\"\n",
    "    initial = tf.constant(0.1, shape=shape)\n",
    "    return tf.Variable(initial, name=name)\n",
    "\n",
    "X1_inputs = tf.placeholder(tf.int64, [None, n_step1], name='X1_input')\n",
    "X2_inputs = tf.placeholder(tf.int64, [None, n_step2], name='X2_input')\n",
    "y_inputs = tf.placeholder(tf.float32, [None, n_class], name='y_input')    \n",
    "\n",
    "def gru_cell():\n",
    "    with tf.name_scope('gru_cell'):\n",
    "        cell = rnn.GRUCell(hidden_size, reuse=tf.get_variable_scope().reuse)\n",
    "    return rnn.DropoutWrapper(cell, output_keep_prob=keep_prob)\n",
    "\n",
    "# with tf.device('/cpu:0'):   # 把 embedding 放在 CPU上\n",
    "embedding = tf.get_variable(name=\"embedding\", shape=W_embedding.shape, \n",
    "                        initializer=tf.constant_initializer(W_embedding), trainable=True)   # fine-tune\n",
    "    \n",
    "def bi_gru(X_inputs):\n",
    "    \"\"\"build the bi-GRU network. 返回个最后一层的隐含状态。\"\"\"\n",
    "    inputs = tf.nn.embedding_lookup(embedding, X_inputs)  \n",
    "    cells_fw = [gru_cell() for _ in range(n_layer)]\n",
    "    cells_bw = [gru_cell() for _ in range(n_layer)]\n",
    "    initial_states_fw = [cell_fw.zero_state(batch_size, tf.float32) for cell_fw in cells_fw]\n",
    "    initial_states_bw = [cell_bw.zero_state(batch_size, tf.float32) for cell_bw in cells_bw] \n",
    "    outputs, _, _ = rnn.stack_bidirectional_dynamic_rnn(cells_fw, cells_bw, inputs, \n",
    "                        initial_states_fw = initial_states_fw, initial_states_bw = initial_states_bw, dtype=tf.float32)\n",
    "    return outputs\n",
    "    \n",
    "\n",
    "def task_specific_attention(inputs, output_size,\n",
    "                            initializer=layers.xavier_initializer(),\n",
    "                            activation_fn=tf.tanh, scope=None):\n",
    "    \"\"\"\n",
    "    Performs task-specific attention reduction, using learned\n",
    "    attention context vector (constant within task of interest).\n",
    "    Args:\n",
    "        inputs: Tensor of shape [batch_size, units, input_size]\n",
    "            `input_size` must be static (known)\n",
    "            `units` axis will be attended over (reduced from output)\n",
    "            `batch_size` will be preserved\n",
    "        output_size: Size of output's inner (feature) dimension\n",
    "    Returns:\n",
    "        outputs: Tensor of shape [batch_size, output_dim].\n",
    "    \"\"\"\n",
    "    assert len(inputs.get_shape()) == 3 and inputs.get_shape()[-1].value is not None\n",
    "    with tf.variable_scope(scope or 'attention') as scope:\n",
    "        # u_w, attention 向量\n",
    "        attention_context_vector = tf.get_variable(name='attention_context_vector',\n",
    "                                                   shape=[output_size],\n",
    "                                                   initializer=initializer,\n",
    "                                                   dtype=tf.float32)\n",
    "        # 全连接层，把 h_i 转为 u_i ， shape= [batch_size, units, input_size] -> [batch_size, units, output_size]\n",
    "        input_projection = layers.fully_connected(inputs, output_size,\n",
    "                                                  activation_fn=activation_fn,\n",
    "                                                  scope=scope)\n",
    "        # 输出 [batch_size, units]\n",
    "        vector_attn = tf.reduce_sum(tf.multiply(input_projection, attention_context_vector), axis=2, keep_dims=True)\n",
    "        attention_weights = tf.nn.softmax(vector_attn, dim=1)   # 输出 [batch_size, units]\n",
    "        tf.summary.histogram('attention_weigths', attention_weights)\n",
    "        weighted_projection = tf.multiply(inputs, attention_weights)   # ???源代码,原文应该是对 hi 加权！！\n",
    "        outputs = tf.reduce_sum(weighted_projection, axis=1)\n",
    "        return outputs\n",
    "\n",
    "    \n",
    "with tf.variable_scope('bi_gru_title'):\n",
    "    word_encoder_title = bi_gru(X1_inputs)      # title 部分输出\n",
    "    output_title = task_specific_attention(word_encoder_title, hidden_size*2)\n",
    "with tf.variable_scope('bi_gru_content'):\n",
    "    word_encoder_content = bi_gru(X2_inputs)    # content 部分输出\n",
    "    output_content = task_specific_attention(word_encoder_content, hidden_size*2)\n",
    "    \n",
    "output = tf.concat([output_title, output_content], axis=1)\n",
    "W_out = weight_variable([hidden_size * 4, n_class], name='Weight_out') \n",
    "tf.summary.histogram('W_out', W_out)\n",
    "b_out = bias_variable([n_class], name='bias_out') \n",
    "tf.summary.histogram('b_out', b_out)\n",
    "y_pred = tf.nn.xw_plus_b(output, W_out, b_out, name='scores')  #每个类别的分数 scores\n",
    "\n",
    "\n",
    "with tf.name_scope('cost'):\n",
    "    cost = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=y_pred, labels=y_inputs))\n",
    "    tf.summary.scalar('cost', cost)\n",
    "    \n",
    "# ***** 优化求解 *******\n",
    "# 获取模型的所有参数\n",
    "tvars = tf.trainable_variables()\n",
    "# 获取损失函数对于每个参数的梯度\n",
    "grads, _ = tf.clip_by_global_norm(tf.gradients(cost, tvars), max_grad_norm)\n",
    "# 优化器\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate=lr)\n",
    "train_op = optimizer.apply_gradients( zip(grads, tvars),\n",
    "    global_step=tf.contrib.framework.get_or_create_global_step())\n",
    "\n",
    "merged = tf.summary.merge_all() # summary\n",
    "train_writer = tf.summary.FileWriter(summary_path + 'train', sess.graph)\n",
    "# test_writer = tf.summary.FileWriter( '../summary/attention-bigru-title/test')\n",
    "print 'Finished creating the bi-gru model.'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 导入数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loading data...\n",
      "finished loading data, time cost 44.5888\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "import sys\n",
    "sys.path.append('..')\n",
    "from data_helpers import BatchGenerator\n",
    "from data_helpers import to_categorical\n",
    "\n",
    "save_path = '../data/'\n",
    "print('loading data...')\n",
    "time0 = time.time()\n",
    "X_title = np.load(save_path+'X_tr_title.npy')\n",
    "X_content = np.load(save_path+'X_tr_content.npy')\n",
    "X = np.hstack([X_title, X_content])\n",
    "y = np.load(save_path+'y_tr.npy')\n",
    "print('finished loading data, time cost %g' % (time.time() - time0))\n",
    "# 划分验证集\n",
    "sample_num = X.shape[0]\n",
    "valid_num = 100000\n",
    "np.random.seed(13)\n",
    "new_index = np.random.permutation(sample_num)\n",
    "X = X[new_index]\n",
    "y = y[new_index]\n",
    "X_valid = X[:valid_num]\n",
    "# y_valid = y[:valid_num]\n",
    "# # X_train = X[valid_num:]\n",
    "# # y_train = y[valid_num:]\n",
    "# X_train = X[valid_num:]\n",
    "# y_train = y[valid_num:]\n",
    "# print('train_num=%d, valid_num=%d' % (X_train.shape[0], X_valid.shape[0]))\n",
    "\n",
    "# # 构建数据生成器\n",
    "# data_train = BatchGenerator(X_train, y_train, shuffle=True)\n",
    "# data_valid = BatchGenerator(X_valid, y_valid, shuffle=False)\n",
    "\n",
    "# print('X_train.shape=', X_train.shape)\n",
    "# print('X_valid.shape=', X_valid.shape)\n",
    "# print('y_train.shape=', y_train.shape)\n",
    "# print('y_valid.shape=', y_valid.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.append('..')\n",
    "from evaluator import score_eval\n",
    "\n",
    "marked_labels_list = data_valid.y.tolist() # 所有的标注结果\n",
    "valid_data_size = data_valid.y.shape[0]\n",
    "def valid_epoch():\n",
    "    \"\"\"Testing or valid.\"\"\"\n",
    "    data_valid._index_in_epoch = 0  # 先指向第一个值\n",
    "    _batch_size = 2000\n",
    "    fetches = [cost, y_pred]   \n",
    "    batch_num = int(valid_data_size / _batch_size)\n",
    "    start_time = time.time()\n",
    "    _costs = 0.0\n",
    "    predict_labels_list = list()  # 所有的预测结果\n",
    "    for i in xrange(batch_num):\n",
    "        X_batch, y_batch = data_valid.next_batch(_batch_size)\n",
    "        X1_batch = X_batch[:, :n_step1]\n",
    "        X2_batch = X_batch[:, n_step1:]\n",
    "        y_batch = to_categorical(y_batch)\n",
    "        feed_dict = {X1_inputs:X1_batch, X2_inputs:X2_batch,  y_inputs:y_batch, lr:1e-5, batch_size:_batch_size, keep_prob:1.0}\n",
    "        _cost, predict_labels = sess.run(fetches, feed_dict)\n",
    "        _costs += _cost    \n",
    "        predict_labels = map(lambda label: label.argsort()[-1:-6:-1], predict_labels) # 取最大的5个下标\n",
    "        predict_labels_list.extend(predict_labels)\n",
    "    predict_label_and_marked_label_list = zip(predict_labels_list, marked_labels_list)\n",
    "    precision, recall, f1 = score_eval(predict_label_and_marked_label_list)\n",
    "    mean_cost = _costs / batch_num\n",
    "    return mean_cost, precision, recall, f1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 训练模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH 1， lr=0.0001， training ...\n",
      "training cost=0.00958835; valid cost=0.00536478; p=0.873054, r=0.371612, f1=0.260662; speed=5646.19 s/epoch\n",
      "EPOCH 2， lr=0.0001， training ...\n",
      "training cost=0.00489762; valid cost=0.00459141; p=1.1958, r=0.487733, f1=0.346433; speed=5685.07 s/epoch\n",
      "('the save path is ', '../ckpt/attention-bigru-title-content/bi-gru.ckpt-2')\n",
      "EPOCH 3， lr=0.0001， training ...\n",
      "training cost=0.00442403; valid cost=0.00437712; p=1.27545, r=0.517992, f1=0.368382; speed=5676.87 s/epoch\n",
      "EPOCH 4， lr=0.0001， training ...\n",
      "training cost=0.0042049; valid cost=0.00426587; p=1.31414, r=0.533722, f1=0.379565; speed=7131.54 s/epoch\n",
      "('the save path is ', '../ckpt/attention-bigru-title-content/bi-gru.ckpt-4')\n",
      "EPOCH 5， lr=0.0001， training ...\n",
      "training cost=0.00406307; valid cost=0.00419162; p=1.33771, r=0.542876, f1=0.386162; speed=6999.3 s/epoch\n",
      "EPOCH 6， lr=0.0001， training ...\n",
      "training cost=0.00395787; valid cost=0.00417651; p=1.35181, r=0.549024, f1=0.390448; speed=5609.62 s/epoch\n",
      "('the save path is ', '../ckpt/attention-bigru-title-content/bi-gru.ckpt-6')\n",
      "EPOCH 7， lr=9e-05， training ...\n",
      "training cost=0.00386814; valid cost=0.00413411; p=1.36022, r=0.552483, f1=0.392899; speed=6153.45 s/epoch\n",
      "EPOCH 8， lr=8.1e-05， training ...\n",
      "training cost=0.00379535; valid cost=0.00411292; p=1.36763, r=0.555741, f1=0.395165; speed=6517.53 s/epoch\n",
      "('the save path is ', '../ckpt/attention-bigru-title-content/bi-gru.ckpt-8')\n",
      "EPOCH 9， lr=7.29e-05， training ...\n",
      "training cost=0.00373369; valid cost=0.00412564; p=1.37056, r=0.557003, f1=0.396047; speed=5605.95 s/epoch\n",
      "EPOCH 10， lr=6.561e-05， training ...\n",
      "training cost=0.00368103; valid cost=0.00412247; p=1.37236, r=0.55802, f1=0.396712; speed=6423.47 s/epoch\n",
      "('the save path is ', '../ckpt/attention-bigru-title-content/bi-gru.ckpt-10')\n",
      "EPOCH 11， lr=5.9049e-05， training ...\n",
      "training cost=0.00363461; valid cost=0.00412971; p=1.37349, r=0.558529, f1=0.397064; speed=5622.6 s/epoch\n",
      "EPOCH 12， lr=5.31441e-05， training ...\n",
      "training cost=0.00359335; valid cost=0.0041432; p=1.37268, r=0.558324, f1=0.396892; speed=6231.39 s/epoch\n",
      "('the save path is ', '../ckpt/attention-bigru-title-content/bi-gru.ckpt-12')\n",
      "EPOCH 13， lr=4.78297e-05， training ...\n",
      "training cost=0.00355639; valid cost=0.00414219; p=1.37254, r=0.55808, f1=0.396757; speed=5641.87 s/epoch\n",
      "EPOCH 14， lr=4.30467e-05， training ...\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-5-893f6cb54b35>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     24\u001b[0m         \u001b[0my_batch\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mto_categorical\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my_batch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     25\u001b[0m         \u001b[0mfeed_dict\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0mX1_inputs\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mX1_batch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mX2_inputs\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mX2_batch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_inputs\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0my_batch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlr\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0m_lr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mtr_batch_size\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkeep_prob\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;36m0.5\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 26\u001b[0;31m         \u001b[0msummary\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_cost\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msess\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfetches\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# the cost is the mean cost of one batch\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     27\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mglobal_step\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0;36m100\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     28\u001b[0m             \u001b[0mtrain_writer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd_summary\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msummary\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mglobal_step\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/common/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    787\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    788\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[0;32m--> 789\u001b[0;31m                          run_metadata_ptr)\n\u001b[0m\u001b[1;32m    790\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    791\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/common/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    995\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    996\u001b[0m       results = self._do_run(handle, final_targets, final_fetches,\n\u001b[0;32m--> 997\u001b[0;31m                              feed_dict_string, options, run_metadata)\n\u001b[0m\u001b[1;32m    998\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    999\u001b[0m       \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/common/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m_do_run\u001b[0;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1130\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1131\u001b[0m       return self._do_call(_run_fn, self._session, feed_dict, fetch_list,\n\u001b[0;32m-> 1132\u001b[0;31m                            target_list, options, run_metadata)\n\u001b[0m\u001b[1;32m   1133\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1134\u001b[0m       return self._do_call(_prun_fn, self._session, handle, feed_dict,\n",
      "\u001b[0;32m/home/common/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m   1137\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1138\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1139\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1140\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1141\u001b[0m       \u001b[0mmessage\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcompat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/common/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m_run_fn\u001b[0;34m(session, feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[1;32m   1119\u001b[0m         return tf_session.TF_Run(session, options,\n\u001b[1;32m   1120\u001b[0m                                  \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_list\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1121\u001b[0;31m                                  status, run_metadata)\n\u001b[0m\u001b[1;32m   1122\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1123\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_prun_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msession\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "decay = 0.90\n",
    "max_epoch = 5\n",
    "max_max_epoch = 16  # 本例中，\n",
    "tr_batch_size = 256 \n",
    "_lr = 1e-4\n",
    "tr_batch_num = int(data_train.y.shape[0] / tr_batch_size)  # 每个 epoch 中包含的 batch 数\n",
    "global_step = 0   # 迭代的步数，2,700,000 / 256, 则每个 epoch 大概是 10000 步\n",
    "sess.run(tf.global_variables_initializer())\n",
    "saver = tf.train.Saver(max_to_keep=10)           # 最多保存的模型数量\n",
    "for epoch in xrange(max_max_epoch):\n",
    "    if epoch > max_epoch:\n",
    "        _lr = _lr * decay\n",
    "    print 'EPOCH %d， lr=%g， training ...' % (epoch+1, _lr)\n",
    "    time0 = time.time()\n",
    "    start_time = time.time()\n",
    "    _costs = 0.0\n",
    "    show_costs = 0.0\n",
    "    for batch in xrange(tr_batch_num): \n",
    "        global_step += 1\n",
    "        fetches = [merged, cost, train_op]\n",
    "        X_batch, y_batch = data_train.next_batch(tr_batch_size)\n",
    "        X1_batch = X_batch[:, :n_step1]\n",
    "        X2_batch = X_batch[:, n_step1:]\n",
    "        y_batch = to_categorical(y_batch)\n",
    "        feed_dict = {X1_inputs:X1_batch, X2_inputs:X2_batch, y_inputs:y_batch, lr:_lr, batch_size:tr_batch_size, keep_prob:0.5}\n",
    "        summary, _cost, _ = sess.run(fetches, feed_dict) # the cost is the mean cost of one batch\n",
    "        if global_step % 100:\n",
    "            train_writer.add_summary(summary, global_step)\n",
    "        _costs += _cost\n",
    "    # 每个 epoch 进行一次验证    \n",
    "    valid_cost, precision, recall, f1 = valid_epoch()  # valid\n",
    "    mean_cost = _costs / tr_batch_num\n",
    "    print('training cost=%g; valid cost=%g; p=%g, r=%g, f1=%g; speed=%g s/epoch' % (\n",
    "        mean_cost, valid_cost, precision, recall, f1, time.time()-time0) )\n",
    "    if (epoch + 1) % 2 == 0:  # 每 2 个 epoch 保存一次模型\n",
    "        save_path = saver.save(sess, model_save_path, global_step=(epoch+1))\n",
    "        print('the save path is ', save_path)           "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "每个 epoch 2899952/256=11327 个batch\n",
    "\n",
    "- Bi-LSTM 模型<br/>\n",
    "**batch_size=256，迭代12个epoch，基本收敛。结果： 验证集 f1=0.38618； 提交结果 0.3873186**\n",
    "- Bi-GRU 模型<br/>\n",
    "**batch_size=256，迭代15个epoch。大概在 13 个 epoch 就收敛了。结果： 验证集 f1=0.390534； 提交结果 0.39198**\n",
    "- attention-Bi-GRU 模型<br/>\n",
    "**batch_size=256，迭代18个epoch。在16个epoch收敛。结果：验证集 f1=0.391734 ；提交结果 0.39310**\n",
    "- attention-Bi-GRU-title-content 模型<br/>\n",
    "**batch_size=256，迭代11个epoch收敛。结果，验证集 f1=0.396892；提交结果 0.399367 **\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 若没有收敛，继续迭代，改成 sgd 优化器"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "add_epoch = 8          # 继续迭代的 epoch 数\n",
    "tr_batch_size = 256    # 把 batch_size 调小一些   \n",
    "_lr = 5e-4\n",
    "for epoch in xrange(max_max_epoch, max_max_epoch+add_epoch):\n",
    "    _lr = _lr * decay\n",
    "    print 'EPOCH %d， lr=%g， training ...' % (epoch+1, _lr)\n",
    "    time0 = time.time()\n",
    "    start_time = time.time()\n",
    "    _costs = 0.0\n",
    "    show_costs = 0.0\n",
    "    for batch in xrange(tr_batch_num): \n",
    "        global_step += 1\n",
    "        fetches = [merged, cost, sgd_train_op]\n",
    "        X_batch, y_batch = data_train.next_batch(tr_batch_size)\n",
    "        X1_batch = X_batch[:, :n_step1]\n",
    "        X2_batch = X_batch[:, n_step1:]\n",
    "        y_batch = to_categorical(y_batch)\n",
    "        feed_dict = {X1_inputs:X1_batch, X2_inputs:X2_batch, y_inputs:y_batch, lr:_lr, batch_size:tr_batch_size, keep_prob:0.5}\n",
    "        summary, _cost, _ = sess.run(fetches, feed_dict) # the cost is the mean cost of one batch\n",
    "        if global_step % 100:\n",
    "            train_writer.add_summary(summary, global_step)\n",
    "        _costs += _cost\n",
    "    # 每个 epoch 进行一次验证    \n",
    "    valid_cost, precision, recall, f1 = valid_epoch()  # valid\n",
    "    mean_cost = _costs / tr_batch_num\n",
    "    print('training cost=%g; valid cost=%g; p=%g, r=%g, f1=%g; speed=%g s/epoch' % (\n",
    "        mean_cost, valid_cost, precision, recall, f1, time.time()-time0) )\n",
    "    if (epoch + 1) % 2 == 0:  # 每 2 个 epoch 保存一次模型\n",
    "        save_path = saver.save(sess, model_save_path, global_step=(epoch+1))\n",
    "        print('the save path is ', save_path)      "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 本地测试"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "begin ...\n",
      "INFO:tensorflow:Restoring parameters from ../ckpt/attention-bigru-title-content/bi-gru.ckpt-12\n",
      "Finished loading model.\n"
     ]
    }
   ],
   "source": [
    "sys.path.append('..')\n",
    "from evaluator import score_eval\n",
    "\n",
    "print('begin ...')\n",
    "# X_valid = np.load('../data/X_valid.npy')\n",
    "model_name = 'attention-bigru-title-content-256'\n",
    "# 保存 本地测试的标注数据\n",
    "marked_labels_list = np.load('../data/marked_labels_list.npy')\n",
    "local_scores_path = '../local_scores/' + model_name + '.npy'\n",
    "\n",
    "# 导入保存好的模型\n",
    "saver = tf.train.Saver()\n",
    "best_model_path = '../ckpt/attention-bigru-title-content/bi-gru.ckpt-12'  # 导入最优模型\n",
    "saver.restore(sess, best_model_path)\n",
    "print('Finished loading model.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "local predicting ...\n",
      "local valid p=1.37269, r=0.558328, f1=0.396895; speed=170.177 s/epoch\n",
      "Writed the scores into ../local_scores/attention-bigru-title-content-256.npy, time 170.583 s\n"
     ]
    }
   ],
   "source": [
    "# 导入测试数据\n",
    "def local_predict(scores_path=local_scores_path):\n",
    "    \"\"\"预测  valid 结果，并保存预测概率 到  scores.csv 文件中。\"\"\"\n",
    "    print('local predicting ...')\n",
    "    time0 = time.time()\n",
    "    X_te = X_valid\n",
    "    n_sample = X_te.shape[0]  # 测试样本数量\n",
    "    _batch_size = 100\n",
    "    fetches = [y_pred]   \n",
    "    predict_labels_list = list()  # 所有的预测结果\n",
    "    predict_scores = list()\n",
    "    for i in xrange(0, n_sample, _batch_size):\n",
    "        start = i\n",
    "        end = start+_batch_size\n",
    "        if end > n_sample:\n",
    "            end = n_sample\n",
    "            _batch_size = end - start\n",
    "        X_batch = X_te[start:end]\n",
    "        X1_batch = X_batch[:, :n_step1]\n",
    "        X2_batch = X_batch[:, n_step1:]\n",
    "        feed_dict = {X1_inputs:X1_batch, X2_inputs:X2_batch, lr:1e-5,\n",
    "                     batch_size:_batch_size, keep_prob:1.0}\n",
    "        predict_labels = sess.run(fetches, feed_dict)[0]\n",
    "        predict_scores.append(predict_labels)\n",
    "        predict_labels = map(lambda label: label.argsort()[-1:-6:-1], predict_labels) # 取最大的5个下标\n",
    "        predict_labels_list.extend(predict_labels)\n",
    "    predict_scores = np.asarray(predict_scores)\n",
    "    predict_label_and_marked_label_list = zip(predict_labels_list, marked_labels_list)\n",
    "    precision, recall, f1 = score_eval(predict_label_and_marked_label_list)\n",
    "    print('local valid p=%g, r=%g, f1=%g; speed=%g s/epoch' % ( precision, recall, f1, time.time()-time0) )\n",
    "    np.save(local_scores_path, predict_scores)\n",
    "    print('Writed the scores into %s, time %g s' % (local_scores_path, time.time()-time0))\n",
    "    \n",
    "local_predict()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 对测试数据进行预测"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ../ckpt/attention-bigru-title-content/bi-gru.ckpt-12\n"
     ]
    }
   ],
   "source": [
    "# 导入保存好的模型\n",
    "saver = tf.train.Saver()\n",
    "best_model_path = model_save_path + '-' +str(11)\n",
    "saver.restore(sess, best_model_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 导入测试数据\n",
    "def predict():\n",
    "    \"\"\"预测测试集结果，并保存到  result.csv 文件中。\"\"\"\n",
    "    print('Predicting ...')\n",
    "    X1_te = np.load('../data/X_te_title.npy')\n",
    "    X2_te = np.load('../data/X_te_content.npy')\n",
    "    X_te = np.hstack([X1_te, X2_te])\n",
    "    n_sample = X_te.shape[0]  # 测试样本数量\n",
    "    _batch_size = 1000\n",
    "    fetches = [y_pred]   \n",
    "    predict_labels_list = list()  # 所有的预测结果\n",
    "    predict_scores = list()\n",
    "    for i in xrange(0, n_sample, _batch_size):\n",
    "        start = i\n",
    "        end = start+_batch_size\n",
    "        if end > n_sample:\n",
    "            end = n_sample\n",
    "            _batch_size = end - start\n",
    "        X_batch = X_te[start:end]\n",
    "        X1_batch = X_batch[:, :n_step1]\n",
    "        X2_batch = X_batch[:, n_step1:]\n",
    "        feed_dict = {X1_inputs:X1_batch, X2_inputs:X2_batch, lr:1e-5, batch_size:_batch_size, keep_prob:1.0}\n",
    "        predict_labels = sess.run(fetches, feed_dict)[0]\n",
    "        predict_scores.append(predict_labels)\n",
    "        predict_labels = map(lambda label: label.argsort()[-1:-6:-1], predict_labels) # 取最大的5个下标\n",
    "        predict_labels_list.extend(predict_labels)\n",
    "    predict_scores = np.asarray(predict_scores)\n",
    "    return predict_labels_list, np.vstack(predict_scores)\n",
    "\n",
    "def write_result(predict_labels_list, result_path):\n",
    "    \"\"\"把结果写到 result.csv 中\"\"\"\n",
    "    eval_question = np.load('../data/eval_question.npy')\n",
    "    with open('../data/sr_topic2id.pkl', 'rb') as inp:\n",
    "        sr_topic2id = pickle.load(inp)\n",
    "        sr_id2topic = pickle.load(inp)\n",
    "    pred_labels = np.asarray(predict_labels_list).reshape([-1])\n",
    "    pred_topics = sr_id2topic[pred_labels].values.reshape([-1, 5])   # 转为 topic\n",
    "    df_result = pd.DataFrame({'question':eval_question, 'tid0': pred_topics[:,0], 'tid1':pred_topics[:, 1],\n",
    "                         'tid2': pred_topics[:,2], 'tid3':pred_topics[:,3],'tid4': pred_topics[:,4]})\n",
    "    df_result.to_csv(result_path, index=False, header=False)\n",
    "    print('Finished writing the result')\n",
    "    return df_result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicting the result, writing into ../result/attention-gru-title-content-256.csv\n",
      "Predicting ...\n",
      "CPU times: user 1min 33s, sys: 10.5 s, total: 1min 44s\n",
      "Wall time: 1min 30s\n",
      "Finished writing the result\n",
      "217360\n",
      "Saving the predict_scores into ../scores/attention-gru-title-content-256.npy\n",
      "(217360, 1999)\n",
      "CPU times: user 0 ns, sys: 844 ms, total: 844 ms\n",
      "Wall time: 799 ms\n",
      "Finished!\n"
     ]
    }
   ],
   "source": [
    "# 预测\n",
    "print('Predicting the result, writing into %s' % result_path)\n",
    "%time predict_labels_list,predict_scores = predict()\n",
    "df_result = write_result(predict_labels_list, result_path=result_path) \n",
    "print(len(df_result))  # 结果应该为 217360\n",
    "print('Saving the predict_scores into %s' % predict_scores_path)\n",
    "print(predict_scores.shape)\n",
    "%time np.save(predict_scores_path, predict_scores)\n",
    "print('Finished!')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(217360, 1999)\n",
      "(1999,)\n",
      "[ -6.42315722  -5.73911715  -7.2089467  ..., -11.32595444 -15.35256863\n",
      "  -8.03899288]\n"
     ]
    }
   ],
   "source": [
    "print(np.vstack(predict_scores).shape)\n",
    "print(predict_scores[0].shape)\n",
    "print(predict_scores[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>question</th>\n",
       "      <th>tid0</th>\n",
       "      <th>tid1</th>\n",
       "      <th>tid2</th>\n",
       "      <th>tid3</th>\n",
       "      <th>tid4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>6215603645409872328</td>\n",
       "      <td>-7506384235581390893</td>\n",
       "      <td>4610596224687453206</td>\n",
       "      <td>-6839713564940654454</td>\n",
       "      <td>2919247920214845195</td>\n",
       "      <td>-5932391056759866388</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>6649324930261961840</td>\n",
       "      <td>-240041917918953337</td>\n",
       "      <td>2858911571784840089</td>\n",
       "      <td>3418451812342379591</td>\n",
       "      <td>2382911985227044227</td>\n",
       "      <td>-7483543763655495143</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-4251899610700378615</td>\n",
       "      <td>2919247920214845195</td>\n",
       "      <td>-7358589937244777363</td>\n",
       "      <td>2816249700493135244</td>\n",
       "      <td>-7270992690764838239</td>\n",
       "      <td>-3315241959305847628</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6213817087034420233</td>\n",
       "      <td>-8655945395761165989</td>\n",
       "      <td>-4966205278807386328</td>\n",
       "      <td>5804619920623030604</td>\n",
       "      <td>7476760589625268543</td>\n",
       "      <td>-6655927395515165363</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-8930652370334418373</td>\n",
       "      <td>3972493657017129406</td>\n",
       "      <td>-8963554618409314978</td>\n",
       "      <td>-1115593437686158905</td>\n",
       "      <td>6018641953300645757</td>\n",
       "      <td>-7790543634407640064</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               question                  tid0                  tid1  \\\n",
       "0   6215603645409872328  -7506384235581390893   4610596224687453206   \n",
       "1   6649324930261961840   -240041917918953337   2858911571784840089   \n",
       "2  -4251899610700378615   2919247920214845195  -7358589937244777363   \n",
       "3   6213817087034420233  -8655945395761165989  -4966205278807386328   \n",
       "4  -8930652370334418373   3972493657017129406  -8963554618409314978   \n",
       "\n",
       "                   tid2                  tid3                  tid4  \n",
       "0  -6839713564940654454   2919247920214845195  -5932391056759866388  \n",
       "1   3418451812342379591   2382911985227044227  -7483543763655495143  \n",
       "2   2816249700493135244  -7270992690764838239  -3315241959305847628  \n",
       "3   5804619920623030604   7476760589625268543  -6655927395515165363  \n",
       "4  -1115593437686158905   6018641953300645757  -7790543634407640064  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_result.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>question</th>\n",
       "      <th>tid0</th>\n",
       "      <th>tid1</th>\n",
       "      <th>tid2</th>\n",
       "      <th>tid3</th>\n",
       "      <th>tid4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>6215603645409872328</td>\n",
       "      <td>4610596224687453206</td>\n",
       "      <td>-6839713564940654454</td>\n",
       "      <td>5818382893362053755</td>\n",
       "      <td>-7506384235581390893</td>\n",
       "      <td>-5932391056759866388</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>6649324930261961840</td>\n",
       "      <td>3418451812342379591</td>\n",
       "      <td>2858911571784840089</td>\n",
       "      <td>-240041917918953337</td>\n",
       "      <td>2382911985227044227</td>\n",
       "      <td>-7483543763655495143</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-4251899610700378615</td>\n",
       "      <td>2919247920214845195</td>\n",
       "      <td>-7358589937244777363</td>\n",
       "      <td>2816249700493135244</td>\n",
       "      <td>-7270992690764838239</td>\n",
       "      <td>-3315241959305847628</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6213817087034420233</td>\n",
       "      <td>5804619920623030604</td>\n",
       "      <td>-4966205278807386328</td>\n",
       "      <td>-8655945395761165989</td>\n",
       "      <td>244937959911721367</td>\n",
       "      <td>7476760589625268543</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-8930652370334418373</td>\n",
       "      <td>3972493657017129406</td>\n",
       "      <td>-8963554618409314978</td>\n",
       "      <td>-1115593437686158905</td>\n",
       "      <td>-6925670792665757873</td>\n",
       "      <td>6018641953300645757</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               question                 tid0                  tid1  \\\n",
       "0   6215603645409872328  4610596224687453206  -6839713564940654454   \n",
       "1   6649324930261961840  3418451812342379591   2858911571784840089   \n",
       "2  -4251899610700378615  2919247920214845195  -7358589937244777363   \n",
       "3   6213817087034420233  5804619920623030604  -4966205278807386328   \n",
       "4  -8930652370334418373  3972493657017129406  -8963554618409314978   \n",
       "\n",
       "                   tid2                  tid3                  tid4  \n",
       "0   5818382893362053755  -7506384235581390893  -5932391056759866388  \n",
       "1   -240041917918953337   2382911985227044227  -7483543763655495143  \n",
       "2   2816249700493135244  -7270992690764838239  -3315241959305847628  \n",
       "3  -8655945395761165989    244937959911721367   7476760589625268543  \n",
       "4  -1115593437686158905  -6925670792665757873   6018641953300645757  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 参考结果\n",
    "df_result.head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 在全部预测正确的情况下，理论值为：f1=0.713933\n",
    "precision=2.50273, recall=0.998873, f1=0.713933"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "在全部预测正确的情况下，理论值为：\n",
      "precision=2.50273, recall=0.998873, f1=0.713933\n"
     ]
    }
   ],
   "source": [
    "# 假设全部正确，f1 值最高能到多少\n",
    "def padding_label(labels):\n",
    "    \"\"\"把所有的label补齐到长度为 5\"\"\"\n",
    "    label_len = len(labels)\n",
    "    if label_len >= 5:\n",
    "        return labels[:5]\n",
    "    return np.hstack([labels, np.zeros(5-label_len, dtype=int) - 1])\n",
    "    \n",
    "\n",
    "marked_labels_list = data_valid.y.tolist() # 所有的标注结果\n",
    "predict_labels_list = map(padding_label, marked_labels_list)\n",
    "predict_label_and_marked_label_list = zip(predict_labels_list, marked_labels_list)\n",
    "precision, recall, f1 = score_eval(predict_label_and_marked_label_list)\n",
    "print '在全部预测正确的情况下，理论值为：'\n",
    "print 'precision=%g, recall=%g, f1=%g' % (precision, recall, f1)"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [conda root]",
   "language": "python",
   "name": "conda-root-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
